An Exploratory Data and Network Analysis of Movies

# Find NA values for gross and budget
movie_metadata <- movie_metadata[!is.na(movie_metadata$gross), ]
movie_metadata <- movie_metadata[!is.na(movie_metadata$budget), ]
dim(movie_metadata)

# Look at all the different types of content ratings
table(movie_metadata$content_rating)

movie_metadata$content_rating[movie_metadata$content_rating == 'M']   <- 'PG' 
movie_metadata$content_rating[movie_metadata$content_rating == 'GP']  <- 'PG' 
movie_metadata$content_rating[movie_metadata$content_rating == 'X']   <- 'NC-17'

movie_metadata$content_rating[movie_metadata$content_rating == 'Approved']  <- 'R' 
movie_metadata$content_rating[movie_metadata$content_rating == 'Not Rated'] <- 'R' 
movie_metadata$content_rating[movie_metadata$content_rating == 'Passed']    <- 'R' 
movie_metadata$content_rating[movie_metadata$content_rating == 'Unrated']   <- 'R' 
movie_metadata$content_rating <- factor(movie_metadata$content_rating)
table(movie_metadata$content_rating)

# Remove rows with blank content ratings
movie_metadata <- movie_metadata[!(movie_metadata$content_rating %in% ""),]

# add profit and return of investment column
movie_metadata <- movie_metadata %>% 
  mutate(profit = gross - budget,
         return_on_investment_perc = (profit/budget)*100)

# Get colour display types of movies
table(movie_metadata$color)

# delete colour
movie_metadata <- subset(movie_metadata, select = -c(color))

# Look at different languages
table(movie_metadata$language)

table(movie_metadata$country)

levels(movie_metadata$country) <- c(levels(movie_metadata$country), "Others")
movie_metadata$country[(movie_metadata$country != 'USA')&(movie_metadata$country != 'UK')] <- 'Others' 
movie_metadata$country <- factor(movie_metadata$country)
table(movie_metadata$country)

genre = movie_metadata['genres']
genre = data.frame(table(genre))
genre = genre[order(genre$Freq,decreasing=TRUE),]

# Top 20 genres with the most movies
ggplot(genre[1:20,], aes(x=reorder(factor(genre), Freq), y=Freq, alpha=Freq)) + 
  geom_bar(stat = "identity", fill="blue") + 
  geom_text(aes(label=Freq),hjust=1.2, size=3.5)+
  xlab("Genre") + 
  ylab("Number of Movies") + 
  ggtitle("Top 20 genres with the most movies") + 
  coord_flip()

head(movie_metadata$genres)

# Split on "|"
genres_split <- str_split(movie_metadata$genres, pattern="[|]", n=2)

# Create Matrix
genres_matrix <- do.call(rbind, strsplit(movie_metadata$genres, '[|]'))

# Dataframe of genres
genres_df <- as.data.frame(genres_matrix)

# Collapse all genres into one column
genres_one_col <- gather(genres_df) %>% 
  select(value)

# Plot frequency of first column
genres_one_col %>%
  group_by(value) %>% 
  tally() %>% 
  filter(n >= 30) %>% 
  ggplot() +
  geom_bar(aes(x = value, y=n), stat="identity") +
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) +
  xlab("Genre") +
  ylab("Frequency") +
  NULL

# Take old genres column away
movie_metadata <- subset(movie_metadata, select = -c(genres))

# Take first column of genres_df and add it to MAIN df
movie_metadata$genre <- genres_df$V1

movie_metadata %>%
 group_by(title_year, genre) %>%
 summarise(count = n()) %>%
 ggplot(aes(title_year, as.factor(genre))) +
 geom_tile(aes(fill=count),colour="white") +
 scale_fill_gradient(low="light blue",high = "dark blue") +
 xlab("Year of Movie") +
 ylab("Genre of Movie") +
 ggtitle("Heat Map of Movie Genres Produced Over the Years") +
  theme(panel.background = element_blank())

# Add column for total facebook likes
movie_metadata$total_facebook_likes <- movie_metadata$director_facebook_likes + 
  movie_metadata$actor_3_facebook_likes + movie_metadata$actor_1_facebook_likes +
  movie_metadata$cast_total_facebook_likes + movie_metadata$actor_2_facebook_likes + 
  movie_metadata$movie_facebook_likes

# creating a data frame containing avg score, avg votes and avg fb likes
score_votes_likes <- movie_metadata %>% 
  group_by(genre) %>%
  summarise(count = n(),
            avg_score = round(mean(imdb_score), 1),
            avg_votes = mean(num_voted_users),
            avg_facebook_likes = mean(total_facebook_likes),
            avg_reviews = mean(num_user_for_reviews)) %>%
  filter(count > 10)

# arranging data frame by average score
arr_score <- arrange(score_votes_likes, desc(avg_facebook_likes))

as.data.frame(score_votes_likes)

movie_metadata %>%
  top_n(15, profit) %>%
  ggplot(aes(x = imdb_score, y = gross/10^6, size = profit/10^6, color = content_rating)) + 
  geom_point() + 
  geom_hline(aes(yintercept = 600)) + 
  geom_vline(aes(xintercept = 7.75)) + 
  geom_text_repel(aes(label = movie_title), size = 4) +
  xlab("Imdb score") + 
  ylab("Gross money earned in million dollars") + 
  ggtitle("Commercial success Vs Critical acclaim") +
  ggplot2::annotate("text", x = 8.5, y = 700, label = "High ratings \n & High gross") +
  theme(plot.title = element_text(hjust = 0.5))

ggcorr(movie_metadata, label = TRUE, label_round = 2, label_size = 2.8, size = 2, hjust = .85) +
  ggtitle("Correlation Heatmap") +
  theme(plot.title = element_text(hjust = 0.5))

options(scipen = 999)

IMDB <- read.csv("../data/real_deal.csv")

#Removing Duplicates
IMDB <- IMDB[!duplicated(IMDB), ]

#Removing White-Spaces and Special Characters
IMDB$movie_title <- gsub("Â", "", as.character(factor(IMDB$movie_title)))
IMDB$movie_title <- str_trim(IMDB$movie_title, side = "right")

IMDB <- subset(IMDB, select = -c(genres))

colSums(sapply(IMDB, is.na))

#Plotting Facebook likes against IMDB score
ggplot(data = IMDB, aes(x = imdb_score, y = movie_facebook_likes)) +
  geom_point() +
  stat_smooth(method="lm")

#Plotting Aspect Ratio against Facebook likes
IMDB %>% 
  filter(aspect_ratio==1.85 | aspect_ratio==2.35) %>%
  mutate(aspect_ratio=as.character((aspect_ratio))) %>%
  ggplot() +
  geom_point(aes(x=aspect_ratio, y=movie_facebook_likes))

#Constructing Top 20
keywords_split <- str_split(IMDB$plot_keywords, pattern="[|]", n=5)

keywords_matrix <- do.call(rbind, strsplit(as.character(IMDB$plot_keywords), "[|]"))

keywords_df <- as.data.frame(keywords_matrix)

names(keywords_df) <- c("one", "two", "three", "four", "five")

keywords_one_col <- gather(keywords_df) %>% 
  select(value)

keywords_one_col_freq <- keywords_one_col %>%
  group_by(value) %>%
  tally()

top_20 <- keywords_one_col_freq %>%
  select(value, n) %>%
  top_n(20)

movies_with_keywords <- data.frame()
IMDB_keyword_movie <- data.frame()

keywords_one_col %>%
  group_by(value) %>% 
  tally() %>% 
  filter(n > 30) %>% 
  ggplot() +
  geom_bar(aes(x = value, y=n), stat="identity") +
  theme(axis.text.x = element_text(angle = 90, hjust = 1)) 

top_20

#Placing Top 20 Words Against Movie Success

IMDB_true_false <- IMDB 
for (keyword in top_20$value) {
  IMDB_true_false <- cbind(IMDB_true_false, ifelse(str_detect(IMDB$plot_keywords, keyword), "TRUE", "FALSE"))
}

for (i in 1:20) {
  reference <- 27
  names(IMDB_true_false)[reference + i] <- top_20$value[i]  
}

truey <- data.frame()
for (keyword in top_20$value) {
  truey <- rbind(truey, IMDB_true_false %>%
                            filter(get(keyword) == TRUE) %>% select(movie_title, gross, imdb_score, movie_facebook_likes, plot_keywords)) %>%
    distinct(movie_title, .keep_all = T)
}



falsy <- data.frame()
for (keyword in top_20$value) {
  falsy <- rbind(falsy, IMDB_true_false %>%
                            filter(get(keyword) == FALSE) %>% select(movie_title, gross, imdb_score, movie_facebook_likes, plot_keywords)) %>%
    distinct(movie_title, .keep_all = T)
}

truey <- truey %>%
  mutate(tri = "Top 20 Word")

falsy <- falsy %>%
  mutate(tri = "NOT Top 20 Word")

truey_falsy <- full_join(falsy, truey, by = c("movie_title", "gross", "imdb_score", "movie_facebook_likes", "plot_keywords"))

truey_falsy <- truey_falsy %>%
  mutate(tri = coalesce(tri.y, tri.x)) %>%
  select(movie_title, gross, imdb_score, movie_facebook_likes, plot_keywords, tri)

truey_falsy <-  truey_falsy %>%
  group_by(tri) %>%
  na.omit() %>%
  mutate(avg = mean(gross))

truey_falsy_graph <- summarise(truey_falsy, avg = mean(gross))

truey_falsy_graph %>%
  ggplot() +
  geom_bar(aes(x = tri, y=avg, fill=tri), stat="identity", position="stack") +
  theme(axis.text.x = element_blank()) 

#Sentiment

keywords_from_split <- data.frame(lapply(keywords_split, "length<-", max(lengths(keywords_split))))

#Creating a list of titles and years seperatored by "|"
titles <-  IMDB %>%
  select(movie_title)
years <- IMDB %>%
  select(title_year)

titles_years_list <- list()

for (x in 1:4998) {
  seperator="|"
  titles_years_list <- c(titles_years_list, paste(titles$movie_title[x], years$title_year[x], sep="|"))
}

#Renaming columns to movie names
movie_name_list <- IMDB$movie_title
names(keywords_from_split) <- titles_years_list

#Adding column to the begining
key_names <- c('key_word_1', 'key_word_2', 'key_word_3', 'key_word_4', 'key_word_5')
keywords_from_split <- cbind(test_keys = key_names, keywords_from_split)

#Moving columns to rows
keywords_from_split <- melt(keywords_from_split, id = "test_keys")

#Creating new column for year
keywords_from_split_matrix <- do.call(rbind, strsplit(as.character(keywords_from_split$variable), "[|]"))
keywords_from_split <- cbind(keywords_from_split, keywords_from_split_matrix)
names(keywords_from_split) <- c("Key #", "Combined", "Keyword", "Title", "Year")
keywords_from_split = keywords_from_split %>%
  select(`Key #`, Title, Year, Keyword)

#Replacing blank cells within keywords to NA's
keywords_from_split$Keyword[keywords_from_split$Keyword == ""] <- NA
#Removing rows containing NA's
keywords_from_split <- na.omit(keywords_from_split)

#All unique years
all_years <- keywords_from_split %>% select(Year)
all_years <- distinct(all_years)
all_years <- na.omit(all_years)
all_years <- all_years %>%
  filter(Year != "NA")

#Function for sentiment per year
yearly_sentiment <- function(year, df) {
  amount <- nrow(df %>%
    select(Year) %>%
    filter(Year == year))
  df <- df %>%
    filter(Year == year)
  sentiments <- get_nrc_sentiment(as.character(df[4]))
  for (i in 1:length(sentiments)) {
    sentiments[i] <- sentiments[i]/amount
  }
  year_sentiment <- cbind(year, sentiments)
  return (year_sentiment)
}

sentiments <- data.frame()

#For-loop to capture all years
for (i in all_years$Year) {
  sentiments <- rbind(sentiments,yearly_sentiment(i, keywords_from_split))
}

#Making years integers
sentiments$year <- strtoi(sentiments$year)

#Sort by year
sentiments <- sentiments[with(sentiments, order(year)), ]
#sentiments <- sentiments[order(year), ]

#Heatmap for sentiments
rnames <- sentiments[,1]
mat_sentiments <- data.matrix(sentiments[,2:ncol(sentiments)])
rownames(mat_sentiments) <- rnames
mat_sentiments <- t(mat_sentiments)

df_sentiment <-  as.data.frame(mat_sentiments)
names_emotions <- c("anger", "anticipation", "disgust","fear","joy","sadness","surprise","trust","negative","positive")

sentiments_graph <- cbind(names_emotions, df_sentiment)

#Run if there is a problem with heatmap
dev.off()

#Heatmap
heatmap.2(mat_sentiments, Rowv=NA, Colv=NA, scale="row", col=colorRampPalette(c("white","darkblue")),  margins=c(5,10), trace = "none")

#Filtering sentiments graph
sentiments_graph_filter <- sentiments %>%
  filter(year >= 2000)

#Other graphs
# Showing trends between sentiment from the 2000's 
ggplot(sentiments_graph_filter, aes(x = as.numeric(year), y = anger)) + 
  geom_point(alpha = 0.5) + 
  geom_line()

ggplot(sentiments_graph_filter, aes(x = as.numeric(year), y = anticipation)) + 
  geom_point(alpha = 0.5) + 
  geom_line()

ggplot(sentiments_graph_filter, aes(x = as.numeric(year), y = disgust)) + 
  geom_point(alpha = 0.5) + 
  geom_line()

ggplot(sentiments_graph_filter, aes(x = as.numeric(year), y = fear)) + 
  geom_point(alpha = 0.5) + 
  geom_line()

ggplot(sentiments_graph_filter, aes(x = as.numeric(year), y = joy)) + 
  geom_point(alpha = 0.5) + 
  geom_line()

ggplot(sentiments_graph_filter, aes(x = as.numeric(year), y = sadness)) + 
  geom_point(alpha = 0.5) + 
  geom_line()

ggplot(sentiments_graph_filter, aes(x = as.numeric(year), y = surprise)) + 
  geom_point(alpha = 0.5) + 
  geom_line()

ggplot(sentiments_graph_filter, aes(x = as.numeric(year), y = trust)) + 
  geom_point(alpha = 0.5) + 
  geom_line()

ggplot(sentiments_graph_filter, aes(x = as.numeric(year), y = negative)) + 
  geom_point(alpha = 0.5) + 
  geom_line()

ggplot(sentiments_graph_filter, aes(x = as.numeric(year), y = positive)) + 
  geom_point(alpha = 0.5) + 
  geom_line()